{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as dsets\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torchviz import make_dot, make_dot_from_trace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle as pkl\n",
    "from collections import defaultdict\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "import json\n",
    "from tqdm import tqdm, tqdm_notebook\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, f1_score, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# %run ../twitter15/twitter15-datapreprocess.ipynb\n",
    "%run ../twitter16/twitter16_text_processing.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter15_label_file = '../twitter16/label.txt'\n",
    "twitter15_text_file = '../twitter16/source_tweets.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_labels(file):\n",
    "    f = open(file,'r')\n",
    "    labels = {}\n",
    "    \n",
    "    raw_data = f.readlines()\n",
    "    \n",
    "    for line in raw_data:\n",
    "        line = line.strip()\n",
    "        line = line.split(':')\n",
    "        labels[int(line[1])] = line[0]\n",
    "    \n",
    "    return labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{656955120626880512: 'false',\n",
       " 615689290706595840: 'true',\n",
       " 613404935003217920: 'false',\n",
       " 731166399389962242: 'unverified',\n",
       " 714598641827246081: 'unverified',\n",
       " 614467824313106432: 'true',\n",
       " 715515982584881152: 'unverified',\n",
       " 693315824132685824: 'non-rumor',\n",
       " 693843042546106369: 'non-rumor',\n",
       " 622891631293935616: 'false',\n",
       " 692630756548591616: 'non-rumor',\n",
       " 693265096278163456: 'non-rumor',\n",
       " 553589051044151296: 'true',\n",
       " 553590835850514433: 'true',\n",
       " 622858454949040128: 'false',\n",
       " 656870311057575936: 'false',\n",
       " 616311563071434753: 'true',\n",
       " 641666167992647681: 'non-rumor',\n",
       " 525060425184858112: 'true',\n",
       " 672513234419638273: 'false',\n",
       " 544382892378714113: 'true',\n",
       " 681824512120324096: 'non-rumor',\n",
       " 620835698514464768: 'false',\n",
       " 626898253900943360: 'false',\n",
       " 618804516578680832: 'false',\n",
       " 672632899921833984: 'false',\n",
       " 553588178687655936: 'true',\n",
       " 594687353937100801: 'false',\n",
       " 613016993692798977: 'false',\n",
       " 663385747177775105: 'false',\n",
       " 766715993385267201: 'non-rumor',\n",
       " 693141729529184256: 'non-rumor',\n",
       " 524966904885428226: 'true',\n",
       " 613393657161510913: 'false',\n",
       " 544374511194632192: 'true',\n",
       " 706665777332621314: 'unverified',\n",
       " 693827886021767169: 'non-rumor',\n",
       " 604354747357859842: 'false',\n",
       " 767709416879644672: 'non-rumor',\n",
       " 693146363685642240: 'non-rumor',\n",
       " 652783670370115585: 'false',\n",
       " 674187250792558592: 'false',\n",
       " 726043971911213057: 'unverified',\n",
       " 524935485370929152: 'true',\n",
       " 730939370765754368: 'unverified',\n",
       " 723365789378584578: 'unverified',\n",
       " 674244561519161344: 'false',\n",
       " 553535829017370625: 'true',\n",
       " 731093044460683264: 'unverified',\n",
       " 693086218276474880: 'non-rumor',\n",
       " 692150118921957376: 'non-rumor',\n",
       " 656830586577883136: 'false',\n",
       " 553543369604210689: 'true',\n",
       " 767814876433571840: 'non-rumor',\n",
       " 649985459389661184: 'true',\n",
       " 553531413459660800: 'true',\n",
       " 693840291992838147: 'non-rumor',\n",
       " 663925546812837888: 'false',\n",
       " 614604054552018944: 'true',\n",
       " 637382230394822656: 'false',\n",
       " 661222523275636736: 'false',\n",
       " 714531423273746432: 'unverified',\n",
       " 675043569367982081: 'false',\n",
       " 544314234541469696: 'true',\n",
       " 651809229842608128: 'unverified',\n",
       " 693140340367187969: 'non-rumor',\n",
       " 553534838880608256: 'true',\n",
       " 633961125126668288: 'false',\n",
       " 544512108885725184: 'true',\n",
       " 552805488631758849: 'true',\n",
       " 645711628046995462: 'false',\n",
       " 614608936491188225: 'true',\n",
       " 693858804279201794: 'non-rumor',\n",
       " 674014933235859456: 'false',\n",
       " 690042063958732800: 'non-rumor',\n",
       " 693041285087756288: 'non-rumor',\n",
       " 620971220301787136: 'false',\n",
       " 656880145769197568: 'false',\n",
       " 674263945172119552: 'true',\n",
       " 612646796355960832: 'false',\n",
       " 544511199702822913: 'true',\n",
       " 650952376954650629: 'unverified',\n",
       " 693441093048766464: 'non-rumor',\n",
       " 553107921081749504: 'true',\n",
       " 666190716448587776: 'false',\n",
       " 681199637387149313: 'non-rumor',\n",
       " 632377165477191680: 'true',\n",
       " 652992524504600576: 'false',\n",
       " 763738618573623297: 'unverified',\n",
       " 672927317191229440: 'false',\n",
       " 757367391202471937: 'unverified',\n",
       " 688819658057740290: 'non-rumor',\n",
       " 692845716826300418: 'non-rumor',\n",
       " 756282236375277568: 'unverified',\n",
       " 692748411481780224: 'non-rumor',\n",
       " 675193315306905600: 'false',\n",
       " 674364257799004160: 'false',\n",
       " 716461257025581056: 'unverified',\n",
       " 662430295254175744: 'false',\n",
       " 544512676643500033: 'true',\n",
       " 638050997340893184: 'false',\n",
       " 692890520960397312: 'non-rumor',\n",
       " 544324444773433348: 'true',\n",
       " 745236050407194624: 'unverified',\n",
       " 673079581520318464: 'false',\n",
       " 691995324752252930: 'non-rumor',\n",
       " 673984270902427650: 'false',\n",
       " 524975705206304769: 'true',\n",
       " 618805892222468096: 'false',\n",
       " 693086478667112448: 'non-rumor',\n",
       " 614595181845839873: 'true',\n",
       " 654371696195993600: 'false',\n",
       " 665358434351509504: 'false',\n",
       " 755439197125767169: 'unverified',\n",
       " 666633171325353989: 'false',\n",
       " 729647367457230850: 'unverified',\n",
       " 726442550266044416: 'unverified',\n",
       " 666070802924617728: 'false',\n",
       " 544512664769396736: 'true',\n",
       " 692106491365605376: 'non-rumor',\n",
       " 727966590084485120: 'unverified',\n",
       " 691789524498845699: 'non-rumor',\n",
       " 767803368081358848: 'non-rumor',\n",
       " 667379734343471104: 'false',\n",
       " 715671763808493569: 'unverified',\n",
       " 653250169752977408: 'false',\n",
       " 656361703664451585: 'unverified',\n",
       " 643103859729166337: 'false',\n",
       " 714560810266132480: 'unverified',\n",
       " 658065957823324160: 'false',\n",
       " 693534670428971008: 'non-rumor',\n",
       " 722885778448121857: 'unverified',\n",
       " 614599619310407680: 'true',\n",
       " 706933939953344514: 'unverified',\n",
       " 691285663522648065: 'non-rumor',\n",
       " 661229627734667264: 'false',\n",
       " 714755546285477888: 'unverified',\n",
       " 672102358516670465: 'non-rumor',\n",
       " 673686233936072704: 'false',\n",
       " 618192748735299584: 'false',\n",
       " 715264793737879553: 'unverified',\n",
       " 614594667225571328: 'true',\n",
       " 693935557009801218: 'non-rumor',\n",
       " 644229386149888001: 'false',\n",
       " 763428684850094080: 'unverified',\n",
       " 671181758692507648: 'false',\n",
       " 692078399012130816: 'non-rumor',\n",
       " 752875379765968897: 'unverified',\n",
       " 626642648179159040: 'false',\n",
       " 742114513726623744: 'unverified',\n",
       " 716457799392342018: 'unverified',\n",
       " 778625026144792577: 'unverified',\n",
       " 614610920782888960: 'true',\n",
       " 715254040289021952: 'unverified',\n",
       " 673872171341447169: 'false',\n",
       " 693167683701968896: 'non-rumor',\n",
       " 724624672604610562: 'unverified',\n",
       " 655432919595548672: 'unverified',\n",
       " 692014905239666688: 'non-rumor',\n",
       " 544520042810200064: 'true',\n",
       " 692803280238419971: 'non-rumor',\n",
       " 664000310856310784: 'false',\n",
       " 742571519105077248: 'unverified',\n",
       " 767541796410839040: 'non-rumor',\n",
       " 641972184412327937: 'true',\n",
       " 651321040119963648: 'false',\n",
       " 662151653462790144: 'false',\n",
       " 693844030589902848: 'non-rumor',\n",
       " 676067381299576832: 'false',\n",
       " 544491151118860289: 'true',\n",
       " 544515538383564801: 'true',\n",
       " 673615400655970304: 'false',\n",
       " 653432261203750912: 'unverified',\n",
       " 641443248909754368: 'false',\n",
       " 614601139422633984: 'true',\n",
       " 524931324763992064: 'true',\n",
       " 707786906189303808: 'unverified',\n",
       " 656047932093763584: 'false',\n",
       " 723025600810766336: 'non-rumor',\n",
       " 656595123590012928: 'false',\n",
       " 615346611955183616: 'false',\n",
       " 763524712853102596: 'unverified',\n",
       " 663817239896821760: 'false',\n",
       " 659462980476637184: 'false',\n",
       " 669259395902152704: 'false',\n",
       " 620367840902782976: 'false',\n",
       " 733242244522725376: 'unverified',\n",
       " 544301453717041152: 'true',\n",
       " 628045645010608128: 'false',\n",
       " 672169954016403456: 'false',\n",
       " 693076140278153217: 'non-rumor',\n",
       " 674314254732931072: 'false',\n",
       " 714555825122107392: 'unverified',\n",
       " 611039775856812032: 'false',\n",
       " 778681502825451520: 'unverified',\n",
       " 525068915068923904: 'true',\n",
       " 740791134146965504: 'unverified',\n",
       " 544504183341064192: 'true',\n",
       " 553476490315431937: 'true',\n",
       " 761573188543229952: 'non-rumor',\n",
       " 673696115305406466: 'false',\n",
       " 651825062174195712: 'false',\n",
       " 633949800761700352: 'false',\n",
       " 761999790892806144: 'non-rumor',\n",
       " 553587013409325058: 'true',\n",
       " 544513524438155264: 'true',\n",
       " 672539897899577344: 'false',\n",
       " 693691456222052352: 'non-rumor',\n",
       " 693648684857323521: 'non-rumor',\n",
       " 655812191233417216: 'unverified',\n",
       " 672219174463275008: 'false',\n",
       " 614628865353351168: 'true',\n",
       " 727854332188577792: 'unverified',\n",
       " 544358564484378624: 'true',\n",
       " 742012307694223361: 'unverified',\n",
       " 707332312724283392: 'unverified',\n",
       " 755447443009916929: 'unverified',\n",
       " 693555965019492352: 'non-rumor',\n",
       " 705092738224525312: 'unverified',\n",
       " 724661834419048448: 'unverified',\n",
       " 757190314880884736: 'unverified',\n",
       " 544278985249550337: 'true',\n",
       " 544367462012432384: 'true',\n",
       " 775672628493357057: 'unverified',\n",
       " 692797856386777089: 'non-rumor',\n",
       " 614645853291155457: 'true',\n",
       " 682916727282270208: 'non-rumor',\n",
       " 725983128444129280: 'unverified',\n",
       " 766808183696351233: 'unverified',\n",
       " 552791196247269378: 'true',\n",
       " 524943490887991296: 'true',\n",
       " 667534186450825216: 'false',\n",
       " 687643002240679936: 'non-rumor',\n",
       " 723511860516016128: 'unverified',\n",
       " 677099574855639044: 'false',\n",
       " 544391533240516608: 'true',\n",
       " 656825206045020160: 'false',\n",
       " 525025279803424768: 'true',\n",
       " 693119705469587456: 'non-rumor',\n",
       " 666051332504207360: 'false',\n",
       " 692929779696275456: 'non-rumor',\n",
       " 650975967146602496: 'unverified',\n",
       " 553544694765215745: 'true',\n",
       " 674080899055546368: 'false',\n",
       " 626770498328895488: 'false',\n",
       " 657007736467525632: 'false',\n",
       " 614626710248534016: 'true',\n",
       " 751536167183613952: 'unverified',\n",
       " 693771953648373760: 'non-rumor',\n",
       " 649903655160815616: 'true',\n",
       " 692082861525078017: 'non-rumor',\n",
       " 614054616154550273: 'false',\n",
       " 674301960787505153: 'false',\n",
       " 658755852199927808: 'unverified',\n",
       " 748543642323783681: 'unverified',\n",
       " 614494170590367744: 'true',\n",
       " 661870323034431489: 'false',\n",
       " 637873886072320001: 'false',\n",
       " 732004388181434368: 'unverified',\n",
       " 689938201193115648: 'non-rumor',\n",
       " 645362146415525888: 'false',\n",
       " 692758581494599681: 'non-rumor',\n",
       " 672632863452299264: 'false',\n",
       " 727187859367546880: 'unverified',\n",
       " 764931593303646208: 'unverified',\n",
       " 525056576038518785: 'true',\n",
       " 672433211604013057: 'false',\n",
       " 747275598347837440: 'unverified',\n",
       " 774833492865593344: 'unverified',\n",
       " 760928376668454912: 'unverified',\n",
       " 728013148788154368: 'unverified',\n",
       " 614618682543616000: 'true',\n",
       " 693060960001597440: 'non-rumor',\n",
       " 693826104633737217: 'non-rumor',\n",
       " 723644048867774464: 'unverified',\n",
       " 742050150307246080: 'unverified',\n",
       " 675064077367005184: 'false',\n",
       " 524926235030589440: 'true',\n",
       " 749286768554438658: 'unverified',\n",
       " 544380742076088320: 'true',\n",
       " 614605997953404929: 'true',\n",
       " 629503919098429440: 'false',\n",
       " 692623941131722752: 'non-rumor',\n",
       " 626546123713474560: 'false',\n",
       " 672906198434209792: 'false',\n",
       " 673615263040798726: 'false',\n",
       " 647169573599449088: 'false',\n",
       " 641050980985999360: 'false',\n",
       " 727179214546456577: 'unverified',\n",
       " 666107476526432256: 'false',\n",
       " 620916279608651776: 'false',\n",
       " 673664899571060736: 'false',\n",
       " 544271284796784640: 'true',\n",
       " 692071267189395457: 'non-rumor',\n",
       " 651486105628463105: 'unverified',\n",
       " 675065047710892033: 'false',\n",
       " 668144671772778497: 'false',\n",
       " 604625816992002049: 'false',\n",
       " 682996350909157376: 'non-rumor',\n",
       " 688752927381581824: 'non-rumor',\n",
       " 693087220459270144: 'non-rumor',\n",
       " 524962142563610625: 'true',\n",
       " 693573781730783232: 'non-rumor',\n",
       " 692083780123783172: 'non-rumor',\n",
       " 627828211003469825: 'false',\n",
       " 693059995013742592: 'non-rumor',\n",
       " 614594195479752704: 'true',\n",
       " 692691444046372865: 'non-rumor',\n",
       " 614593989203886080: 'true',\n",
       " 690376107825192960: 'non-rumor',\n",
       " 658259426172891136: 'false',\n",
       " 676586804242309121: 'unverified',\n",
       " 651959206287908868: 'false',\n",
       " 681147789653356544: 'true',\n",
       " 652882609219833856: 'false',\n",
       " 544476808566276097: 'true',\n",
       " 687274510643511296: 'non-rumor',\n",
       " 647464349611589632: 'false',\n",
       " 726086935903494144: 'unverified',\n",
       " 691632238035886081: 'non-rumor',\n",
       " 524952883343925249: 'true',\n",
       " 688021039322894336: 'non-rumor',\n",
       " 707308274270539777: 'unverified',\n",
       " 691678576018657281: 'non-rumor',\n",
       " 692874200927698945: 'non-rumor',\n",
       " 666640008149925893: 'false',\n",
       " 663515735231062016: 'false',\n",
       " 643139873264812032: 'false',\n",
       " 689860942671130624: 'non-rumor',\n",
       " 656662726014599168: 'false',\n",
       " 674229534888185856: 'false',\n",
       " 716424773216022530: 'unverified',\n",
       " 672432930426372096: 'false',\n",
       " 600451916414484480: 'false',\n",
       " 692838430783332357: 'non-rumor',\n",
       " 717081129627553792: 'unverified',\n",
       " 765612862681255936: 'non-rumor',\n",
       " 641088973717110784: 'false',\n",
       " 652349108653551616: 'unverified',\n",
       " 777710439870455808: 'unverified',\n",
       " 553508098825261056: 'true',\n",
       " 762793077509464064: 'non-rumor',\n",
       " 614628136634949632: 'true',\n",
       " 580324027715063808: 'true',\n",
       " 634665777400950784: 'false',\n",
       " 674301413040758785: 'false',\n",
       " 613362193787129860: 'false',\n",
       " 743764020679741440: 'unverified',\n",
       " 692820029159653377: 'non-rumor',\n",
       " 693471161313816576: 'non-rumor',\n",
       " 716092408920936448: 'unverified',\n",
       " 524941132237910016: 'true',\n",
       " 727116900983934976: 'unverified',\n",
       " 655986243042459648: 'false',\n",
       " 544283772569788416: 'true',\n",
       " 693818562981421056: 'non-rumor',\n",
       " 658938136299511808: 'unverified',\n",
       " 618449248179191808: 'false',\n",
       " 693897857376632832: 'non-rumor',\n",
       " 692410832307818497: 'non-rumor',\n",
       " 662381914842603520: 'false',\n",
       " 615494435074363392: 'true',\n",
       " 763520953619918848: 'unverified',\n",
       " 766568413418356736: 'non-rumor',\n",
       " 676367888543031296: 'true',\n",
       " 692758050378256388: 'non-rumor',\n",
       " 642465192408940544: 'true',\n",
       " 612438528803213312: 'false',\n",
       " 613425834301485056: 'false',\n",
       " 714567472712589312: 'unverified',\n",
       " 693281966846808064: 'non-rumor',\n",
       " 659428447459110912: 'false',\n",
       " 672902686380003328: 'false',\n",
       " 659439879701602304: 'false',\n",
       " 553590459688570880: 'true',\n",
       " 525058976376193024: 'true',\n",
       " 669201837187334144: 'false',\n",
       " 652300118205427716: 'unverified',\n",
       " 693203974388895744: 'non-rumor',\n",
       " 747443219487678464: 'unverified',\n",
       " 755548076438196225: 'unverified',\n",
       " 728627623488786433: 'unverified',\n",
       " 553476880339599360: 'true',\n",
       " 749039299677417472: 'unverified',\n",
       " 687537772970684417: 'non-rumor',\n",
       " 725070832494624769: 'unverified',\n",
       " 728038172270198787: 'unverified',\n",
       " 500378223977721856: 'true',\n",
       " 724348906096590849: 'unverified',\n",
       " 658786161733734400: 'false',\n",
       " 728207861050970112: 'unverified',\n",
       " 651402689352351744: 'false',\n",
       " 634943791934406657: 'false',\n",
       " 544319832486064128: 'true',\n",
       " 693606934050684928: 'non-rumor',\n",
       " 666810213274689537: 'false',\n",
       " 692702295281262593: 'non-rumor',\n",
       " 692498490249891842: 'non-rumor',\n",
       " 689867195657101312: 'non-rumor',\n",
       " 553512735192141826: 'true',\n",
       " 544517264054423552: 'true',\n",
       " 656629493377990656: 'false',\n",
       " 663744139666804736: 'unverified',\n",
       " 675005503160901632: 'false',\n",
       " 638047610973089793: 'false',\n",
       " 727623131494387714: 'unverified',\n",
       " 723198441690636288: 'non-rumor',\n",
       " 641082932740947972: 'true',\n",
       " 648894687542034432: 'false',\n",
       " 692925396292091905: 'non-rumor',\n",
       " 752965545528528898: 'unverified',\n",
       " 778572032531427332: 'unverified',\n",
       " 716451800581279744: 'unverified',\n",
       " 637868242560638980: 'false',\n",
       " 498430783699554305: 'true',\n",
       " 641430951403343872: 'false',\n",
       " 635632641635667968: 'false',\n",
       " 652000523525091328: 'unverified',\n",
       " 626897206717624320: 'false',\n",
       " 693476774462820352: 'non-rumor',\n",
       " 626516248206135296: 'false',\n",
       " 676094756758364160: 'false',\n",
       " 715256242990505984: 'unverified',\n",
       " 690650123358093312: 'non-rumor',\n",
       " 701514249269542912: 'unverified',\n",
       " 623599854661541888: 'true',\n",
       " 629209452793626624: 'false',\n",
       " 693811101146963968: 'non-rumor',\n",
       " 666497286663503872: 'false',\n",
       " 642432477185867776: 'false',\n",
       " 682999206290829312: 'non-rumor',\n",
       " 660466342038867969: 'false',\n",
       " 656202544331517952: 'false',\n",
       " 727172374999666688: 'unverified',\n",
       " 647193820812177408: 'false',\n",
       " 723772395211862016: 'unverified',\n",
       " 614607711368519680: 'true',\n",
       " 656818921979310081: 'false',\n",
       " 647168914955243520: 'false',\n",
       " 661251968078221312: 'false',\n",
       " 690680149164085248: 'non-rumor',\n",
       " 665361505303584774: 'false',\n",
       " 613294443878305796: 'false',\n",
       " 676120162018451456: 'false',\n",
       " 665379967757324288: 'false',\n",
       " 766752508312166402: 'non-rumor',\n",
       " 612741808125120513: 'false',\n",
       " 689267711856263168: 'non-rumor',\n",
       " 613023744454475776: 'false',\n",
       " 614671961801785344: 'true',\n",
       " 675490515748425728: 'true',\n",
       " 740748123581087745: 'unverified',\n",
       " 690580180805509121: 'non-rumor',\n",
       " 552984502063337472: 'true',\n",
       " 650128194209730561: 'true',\n",
       " 612841482823729152: 'false',\n",
       " 636925368927064064: 'false',\n",
       " 674015148382666752: 'false',\n",
       " 656834590779289600: 'false',\n",
       " 681767380305985536: 'false',\n",
       " 648310692794109952: 'false',\n",
       " 524995771587108864: 'true',\n",
       " 692752491482615808: 'non-rumor',\n",
       " 693502060545703937: 'non-rumor',\n",
       " 766789709045518336: 'unverified',\n",
       " 652982112870662144: 'unverified',\n",
       " 693456187036037123: 'non-rumor',\n",
       " 692796451987034113: 'non-rumor',\n",
       " 703234354579898368: 'unverified',\n",
       " 693466451081060353: 'non-rumor',\n",
       " 655815788675399680: 'unverified',\n",
       " 552832817089236992: 'true',\n",
       " 544291965513134080: 'true',\n",
       " 726190016435728385: 'unverified',\n",
       " 676870737932742656: 'false',\n",
       " 672553813249826816: 'non-rumor',\n",
       " 649881917534433280: 'true',\n",
       " 692023855926374400: 'non-rumor',\n",
       " 725174535897620481: 'unverified',\n",
       " 688446106943008768: 'non-rumor',\n",
       " 640107106943766528: 'false',\n",
       " 748640007934590976: 'unverified',\n",
       " 724320681517670400: 'unverified',\n",
       " 640644123339431936: 'false',\n",
       " 661299012386095105: 'false',\n",
       " 613061089518034944: 'false',\n",
       " 672424512516964352: 'false',\n",
       " 628319096606732290: 'true',\n",
       " 524936872666353664: 'true',\n",
       " 673611867181473792: 'false',\n",
       " 673581371458199552: 'false',\n",
       " 668872645589471232: 'false',\n",
       " 669613420665159680: 'false',\n",
       " 676718762830221312: 'false',\n",
       " 613194145423826944: 'false',\n",
       " 701175292937707520: 'unverified',\n",
       " 544289409294553088: 'true',\n",
       " 665575674485211136: 'false',\n",
       " 641191076716417025: 'false',\n",
       " 682895395077373952: 'non-rumor',\n",
       " 553587303172833280: 'true',\n",
       " 633823543541768192: 'false',\n",
       " 544381485591982083: 'true',\n",
       " 766323684076322816: 'non-rumor',\n",
       " 691608761128067072: 'non-rumor',\n",
       " 614638036593299456: 'true',\n",
       " 553587672137334785: 'true',\n",
       " 552811386259386370: 'true',\n",
       " 672828403016429568: 'false',\n",
       " 604642913188872192: 'false',\n",
       " 765359928361922560: 'non-rumor',\n",
       " 626739062792032256: 'false',\n",
       " 663722803489808384: 'false',\n",
       " 691719802499432448: 'non-rumor',\n",
       " 661120256732041216: 'false',\n",
       " 634404792241143809: 'false',\n",
       " 524923676484177920: 'true',\n",
       " 665364878287224834: 'false',\n",
       " 642027433693171712: 'false',\n",
       " 626901072209121282: 'false',\n",
       " 692735698349199360: 'non-rumor',\n",
       " 544277117039837184: 'true',\n",
       " 663904307113275392: 'false',\n",
       " 544358533819420672: 'true',\n",
       " 693528374996652032: 'non-rumor',\n",
       " 724703995147751424: 'unverified',\n",
       " 759460236042305536: 'unverified',\n",
       " 524965775036387329: 'true',\n",
       " 673346166822694914: 'false',\n",
       " 525025463648137216: 'true',\n",
       " 723521076446142465: 'unverified',\n",
       " 672488384695279616: 'false',\n",
       " 642145732435292160: 'false',\n",
       " 757748522481491968: 'unverified',\n",
       " 716439952922312704: 'unverified',\n",
       " 614604580601597953: 'true',\n",
       " 643225207705075716: 'false',\n",
       " 654351281260113920: 'false',\n",
       " 692540218310803456: 'non-rumor',\n",
       " 661102820976930816: 'false',\n",
       " 693061713042771968: 'non-rumor',\n",
       " 544518335019229184: 'true',\n",
       " 614624331717545984: 'true',\n",
       " 615840865815298048: 'true',\n",
       " 645356735545364480: 'false',\n",
       " 693651239486263296: 'non-rumor',\n",
       " 544274934835707905: 'true',\n",
       " 655080266784968705: 'false',\n",
       " 743058135300988932: 'unverified',\n",
       " 658462819667615744: 'false',\n",
       " 614617234942656512: 'true',\n",
       " 778949749156245504: 'unverified',\n",
       " 715255507506892800: 'unverified',\n",
       " 544271362022338560: 'true',\n",
       " 693321561542127616: 'non-rumor',\n",
       " 692082592380751874: 'non-rumor',\n",
       " 552793679082311680: 'true',\n",
       " 693129259334909952: 'non-rumor',\n",
       " 648993731169939456: 'unverified',\n",
       " 693893182338289664: 'non-rumor',\n",
       " 676955015265837056: 'non-rumor',\n",
       " 651786568592658433: 'unverified',\n",
       " 544319274072817664: 'true',\n",
       " 640182854928961536: 'unverified',\n",
       " 727588444000526336: 'unverified',\n",
       " 628681462976528384: 'true',\n",
       " 724044187113512960: 'unverified',\n",
       " 650037735579910145: 'true',\n",
       " 544289941996326912: 'true',\n",
       " 649974380416618496: 'true',\n",
       " 688428083540553728: 'non-rumor',\n",
       " 701975210044497921: 'unverified',\n",
       " 692826642583019521: 'non-rumor',\n",
       " 654343805081157633: 'false',\n",
       " 742055437932040193: 'unverified',\n",
       " 616481759329427456: 'true',\n",
       " 613114185505996802: 'false',\n",
       " 552785375161499649: 'true',\n",
       " 697992796565741569: 'unverified',\n",
       " 714577521992343552: 'unverified',\n",
       " 760120409429643266: 'unverified',\n",
       " 692368829368918017: 'non-rumor',\n",
       " 672180526946656256: 'false',\n",
       " 687409983139495939: 'non-rumor',\n",
       " 767092161716350980: 'non-rumor',\n",
       " 693857080193761285: 'non-rumor',\n",
       " 623818627054206977: 'false',\n",
       " 757450526153900032: 'unverified',\n",
       " 728631482722308096: 'unverified',\n",
       " 693402634011701248: 'non-rumor',\n",
       " 667465205258051584: 'false',\n",
       " 524925215235911680: 'true',\n",
       " 525028734991343617: 'true',\n",
       " 647169726158925824: 'false',\n",
       " 780436430732525569: 'unverified',\n",
       " 723915846985342976: 'unverified',\n",
       " 631388553801564160: 'false',\n",
       " 656811196218286080: 'false',\n",
       " 674358835675549696: 'true',\n",
       " 766541461189832704: 'non-rumor',\n",
       " 616765822095261700: 'true',\n",
       " 693857551620968448: 'non-rumor',\n",
       " 614610364702044164: 'true',\n",
       " 693648675772436480: 'non-rumor',\n",
       " 524932935137628160: 'true',\n",
       " 640967537178472448: 'true',\n",
       " 692884886374318081: 'non-rumor',\n",
       " 614595106906226688: 'true',\n",
       " 693181857261719553: 'non-rumor',\n",
       " 693200309955469312: 'non-rumor',\n",
       " 525049639016615937: 'true',\n",
       " 755475529294352385: 'unverified',\n",
       " 692516224245235712: 'non-rumor',\n",
       " 707604196812591104: 'unverified',\n",
       " 614790312955904000: 'true',\n",
       " 765739657908850688: 'unverified',\n",
       " 687748306571804672: 'non-rumor',\n",
       " 544520273718812672: 'true',\n",
       " 748558349139058688: 'unverified',\n",
       " 544350712365207552: 'true',\n",
       " 688741019979001856: 'non-rumor',\n",
       " 614702244001382400: 'true',\n",
       " 693641113693884416: 'non-rumor',\n",
       " 767861179951648768: 'non-rumor',\n",
       " 524925987239120897: 'true',\n",
       " 666335363099660288: 'non-rumor',\n",
       " 692550140754841603: 'non-rumor',\n",
       " 688755462783782912: 'non-rumor',\n",
       " 676491949524713473: 'non-rumor',\n",
       " 693939356390653952: 'non-rumor',\n",
       " 649584608560832513: 'unverified',\n",
       " 707321278135463936: 'unverified',\n",
       " 716448280201269248: 'unverified',\n",
       " 525019752507658240: 'true',\n",
       " 745365403237376000: 'unverified',\n",
       " 690106189951139840: 'non-rumor',\n",
       " 767698691205410816: 'non-rumor',\n",
       " 614620298877513729: 'true',\n",
       " 654336219447472128: 'unverified',\n",
       " 552791578893619200: 'true',\n",
       " 614599587362414592: 'true',\n",
       " 701872343736520704: 'unverified',\n",
       " 693571567377342466: 'non-rumor',\n",
       " 649889459836710912: 'true',\n",
       " 763430916236640256: 'unverified',\n",
       " 688366079396163584: 'non-rumor',\n",
       " 524980744658382848: 'true',\n",
       " 711994705333112832: 'unverified',\n",
       " 544352727971954690: 'true',\n",
       " 765779159176077312: 'non-rumor',\n",
       " 690969315470868480: 'non-rumor',\n",
       " 705093513076015104: 'unverified',\n",
       " 755523157700648960: 'unverified',\n",
       " 742678930516123650: 'unverified',\n",
       " 614594259900080128: 'true',\n",
       " 614614133410033664: 'true',\n",
       " 544278335455776769: 'true',\n",
       " 665309822208729088: 'non-rumor',\n",
       " 732971411157880832: 'unverified',\n",
       " 693466724822323200: 'non-rumor',\n",
       " 651018580989972480: 'true',\n",
       " 727982226290442240: 'unverified',\n",
       " 667073349974102017: 'true',\n",
       " 524922729485848576: 'true',\n",
       " 668849913678209024: 'true',\n",
       " 692743102432346113: 'non-rumor',\n",
       " 778299287293816832: 'unverified',\n",
       " 714503380476026880: 'unverified',\n",
       " 689915847939219456: 'non-rumor',\n",
       " 728625967921401856: 'unverified',\n",
       " 524925050739490816: 'true',\n",
       " 628336948810268673: 'true',\n",
       " 689890946683576322: 'non-rumor',\n",
       " 767459987476054016: 'non-rumor',\n",
       " 767203096472719364: 'non-rumor',\n",
       " 689641419825242112: 'non-rumor',\n",
       " 692393375891361794: 'non-rumor',\n",
       " 552783667052167168: 'true',\n",
       " 552834961762709505: 'true',\n",
       " 716416753409081344: 'unverified',\n",
       " 707420972173955072: 'unverified',\n",
       " 780882510645370880: 'unverified',\n",
       " 553160652567498752: 'true',\n",
       " 544268732046913536: 'true',\n",
       " 767855091109863424: 'non-rumor',\n",
       " 692911086920667136: 'non-rumor',\n",
       " 544282005941530624: 'true',\n",
       " 614609560658161664: 'true',\n",
       " 580348081100734464: 'true',\n",
       " 723504069814444033: 'unverified',\n",
       " 553586860334010368: 'true',\n",
       " 640118021101604864: 'unverified',\n",
       " 688751061503442944: 'non-rumor',\n",
       " 778530869208190976: 'unverified',\n",
       " 544291804057960448: 'true',\n",
       " 688004802245214208: 'non-rumor',\n",
       " 743126914794037248: 'unverified',\n",
       " 653432177632387072: 'unverified',\n",
       " 628604055644934144: 'true',\n",
       " 692163879909064704: 'non-rumor',\n",
       " 728273551883534336: 'unverified',\n",
       " 524942470472548352: 'true',\n",
       " 552792544132997121: 'true',\n",
       " 727834854931435522: 'unverified',\n",
       " 544289311504355328: 'true',\n",
       " 614494460747997184: 'true',\n",
       " 615592870540488704: 'true',\n",
       " 689661418484994049: 'non-rumor',\n",
       " 724603564946010112: 'unverified',\n",
       " 714598318333042688: 'unverified',\n",
       " 656848877463560192: 'non-rumor',\n",
       " 692157602554343424: 'non-rumor',\n",
       " 765735409984806912: 'unverified',\n",
       " 614610102927147008: 'true',\n",
       " 614650850393391104: 'true',\n",
       " 614648099542204416: 'true',\n",
       " 544514564407427072: 'true',\n",
       " 707305999971921920: 'unverified',\n",
       " 644329878876237824: 'non-rumor',\n",
       " 553566026030272512: 'true',\n",
       " 715253497462145024: 'unverified',\n",
       " 693804277031157761: 'non-rumor',\n",
       " 715268937873760256: 'unverified',\n",
       " 763964234774347776: 'non-rumor',\n",
       " 614616994499788800: 'true',\n",
       " 693185853867294721: 'non-rumor',\n",
       " 553506608203169792: 'true',\n",
       " 751856580874960897: 'unverified',\n",
       " 712223438698627073: 'unverified',\n",
       " 707405809181917184: 'unverified',\n",
       " 693013768738115584: 'non-rumor',\n",
       " 744390771869102080: 'unverified',\n",
       " 524959809402331137: 'true',\n",
       " 778689027918618625: 'unverified',\n",
       " 707412869860696064: 'unverified',\n",
       " 650710483289419776: 'unverified',\n",
       " 656523458139045889: 'unverified',\n",
       " 693135382309896192: 'non-rumor',\n",
       " 623533663947517952: 'true',\n",
       " 692833140428017664: 'non-rumor',\n",
       " 614599815280857088: 'true',\n",
       " 693853310479155200: 'non-rumor',\n",
       " 614615157730357248: 'true',\n",
       " 714851393861881857: 'unverified',\n",
       " 767859423150759936: 'non-rumor',\n",
       " 764906140257837056: 'non-rumor',\n",
       " 619242759359037440: 'true',\n",
       " 552792913910833152: 'true',\n",
       " 764368020391223296: 'unverified',\n",
       " 741995157969592321: 'unverified',\n",
       " 767754921403899906: 'non-rumor',\n",
       " 692940257134653441: 'non-rumor',\n",
       " 667487418388488192: 'true',\n",
       " 553586897168392192: 'true',\n",
       " 730516765525082112: 'unverified',\n",
       " 693128224960856064: 'non-rumor',\n",
       " 553221600955621376: 'true',\n",
       " 693071538896162816: 'non-rumor',\n",
       " 693869818366287872: 'non-rumor',\n",
       " 714998347111669760: 'unverified',\n",
       " 686666933949837312: 'non-rumor',\n",
       " 767710042816602112: 'non-rumor',\n",
       " 714811995573325828: 'unverified',\n",
       " 728101712762834944: 'unverified',\n",
       " 544399927045283840: 'true',\n",
       " 524947674164760577: 'true',\n",
       " 701539698452393986: 'unverified',\n",
       " 653285570383335424: 'unverified',\n",
       " 552792802309181440: 'true',\n",
       " 707310135291416576: 'unverified',\n",
       " 693196937172951040: 'non-rumor',\n",
       " 614891047886434304: 'true',\n",
       " 525023025792835585: 'true',\n",
       " 765887221736046592: 'unverified',\n",
       " 767561970182598657: 'non-rumor',\n",
       " 687984820815851521: 'non-rumor',\n",
       " 544282227035869184: 'true',\n",
       " 766822417712963584: 'non-rumor',\n",
       " 553558982476828674: 'true',\n",
       " 544290258951892992: 'true',\n",
       " 650254360727973889: 'unverified',\n",
       " 544329935943237632: 'true',\n",
       " 693586539562037248: 'non-rumor',\n",
       " 764497123530375169: 'non-rumor',\n",
       " 724939017410727938: 'unverified',\n",
       " 651044059222556674: 'unverified',\n",
       " 552802654641225728: 'true',\n",
       " 723477822950395904: 'unverified',\n",
       " 544305540286148609: 'true',\n",
       " 693906624776245249: 'non-rumor',\n",
       " 524923462398513152: 'true',\n",
       " 763896522790440960: 'unverified',\n",
       " 616421546702336000: 'true',\n",
       " 524944399890124801: 'true',\n",
       " 686657138270277634: 'non-rumor',\n",
       " 544391176137089024: 'true',\n",
       " 524969201102901248: 'true',\n",
       " 756690088533393409: 'unverified',\n",
       " 767807335691653120: 'non-rumor',\n",
       " 552806757672964097: 'true',\n",
       " 693761289601060864: 'non-rumor',\n",
       " 544350567183556608: 'true',\n",
       " 758825535480864769: 'unverified',\n",
       " 707300612862566400: 'unverified',\n",
       " 693080409282846720: 'non-rumor',\n",
       " 614593386188828672: 'true',\n",
       " 692665281362202624: 'non-rumor',\n",
       " 767129248045948929: 'non-rumor',\n",
       " 692701223326056449: 'non-rumor',\n",
       " 732981604826677249: 'unverified',\n",
       " 692753210692476928: 'non-rumor',\n",
       " 650046859537448960: 'true',\n",
       " 693171092555431936: 'non-rumor',\n",
       " 693546915892428800: 'non-rumor',\n",
       " 544269749405097984: 'true',\n",
       " 760109079133990912: 'unverified',\n",
       " 779633844680962048: 'unverified',\n",
       " 765859710503378944: 'non-rumor'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter15_labels = load_labels(twitter15_label_file)\n",
    "twitter15_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Structures so that Pickle can work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Node:\n",
    "    def __init__(self,uid,tid,time_stamp,label):\n",
    "        self.children = {}\n",
    "        self.childrenList = []\n",
    "        self.num_children = 0\n",
    "        self.tid = tid\n",
    "        self.uid = uid\n",
    "        self.label = label\n",
    "        self.time_stamp = time_stamp\n",
    "    \n",
    "    def add_child(self,node):\n",
    "        if node.uid not in self.children:\n",
    "            self.children[node.uid] = node\n",
    "            self.num_children += 1\n",
    "        else:\n",
    "            self.children[node.uid] = node\n",
    "        self.childrenList = list(self.children.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Tree:\n",
    "    def __init__(self,root):\n",
    "        self.root = root\n",
    "        self.tweet_id = root.tid\n",
    "        self.uid = root.uid\n",
    "        self.height = 0\n",
    "        self.nodes = 0\n",
    "    \n",
    "    def show(self):\n",
    "        queue = [self.root,0]\n",
    "        \n",
    "        while len(queue) != 0:\n",
    "            toprint = queue.pop(0)\n",
    "            if toprint == 0:\n",
    "                print('\\n')\n",
    "            else:\n",
    "                print(toprint.uid,end=' ')\n",
    "                queue += toprint.children.values()\n",
    "                queue.append(0)\n",
    "                \n",
    "    def insertnode(self,curnode,parent,child):\n",
    "        if curnode.uid == parent.uid:\n",
    "            curnode.add_child(child)\n",
    "            return 1\n",
    "\n",
    "        elif parent.uid in curnode.children:\n",
    "            s = self.insertnode(curnode.children[parent.uid],parent,child)\n",
    "            return 2\n",
    "        else:\n",
    "            for node in curnode.children:\n",
    "                s = self.insertnode(curnode.children[node],parent,child)\n",
    "                if s == 2:\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadPklFileNum(datapath,incSize,fileNum):\n",
    "    \n",
    "    with open(datapath+str(incSize)+'inc_'+str(fileNum)+'.pickle', 'rb') as handle:\n",
    "        twitTrees = pkl.load(handle)\n",
    "    return twitTrees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadTreeFilesOfIncrement(datapath,incSize):\n",
    "    twittertrees = {}\n",
    "    \n",
    "    files = [x for x in os.listdir(t15Datapath) if str(incSize)+'inc' in x]\n",
    "    \n",
    "    for file in tqdm(files):\n",
    "        with open(datapath+file,'rb') as handle:\n",
    "            partialTrees = pkl.load(handle)\n",
    "        twittertrees.update(partialTrees)\n",
    "        \n",
    "    return twittertrees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "t15Datapath = '/home/nikhil.pinnaparaju/Research/Temporal Tree Encoding/twitter16/pickledTrees/'\n",
    "# twitter15_trees = loadPklFileNum(t15Datapath,20,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e020aee9ea02404eaab58e50487d4a2e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=9), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "twitter15_trees = loadTreeFilesOfIncrement(t15Datapath,20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X = []\n",
    "y = []\n",
    "for tid in twitter15_trees:\n",
    "    if tid in twitter15_trees and tid in twitter15_labels:\n",
    "        X.append(twitter15_trees[tid])\n",
    "        y.append(twitter15_labels[tid])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "31"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[0][-1].root.num_children"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    device = 'cuda:2'\n",
    "    device = 'cpu'\n",
    "else:\n",
    "    device = 'cpu'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading UserData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:00<00:00, 9140.35it/s]\n",
      "  0%|          | 0/33 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n",
      "150\n",
      "151\n",
      "152\n",
      "153\n",
      "154\n",
      "155\n",
      "156\n",
      "157\n",
      "158\n",
      "159\n",
      "160\n",
      "161\n",
      "162\n",
      "163\n",
      "164\n",
      "165\n",
      "166\n",
      "167\n",
      "168\n",
      "169\n",
      "170\n",
      "171\n",
      "172\n",
      "173\n",
      "174\n",
      "175\n",
      "176\n",
      "177\n",
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "182\n",
      "183\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n",
      "193\n",
      "194\n",
      "195\n",
      "196\n",
      "197\n",
      "198\n",
      "199\n",
      "200\n",
      "201\n",
      "202\n",
      "203\n",
      "204\n",
      "205\n",
      "206\n",
      "207\n",
      "208\n",
      "209\n",
      "210\n",
      "211\n",
      "212\n",
      "213\n",
      "214\n",
      "215\n",
      "216\n",
      "217\n",
      "218\n",
      "219\n",
      "220\n",
      "221\n",
      "222\n",
      "223\n",
      "224\n",
      "225\n",
      "226\n",
      "227\n",
      "228\n",
      "229\n",
      "230\n",
      "231\n",
      "232\n",
      "233\n",
      "234\n",
      "235\n",
      "236\n",
      "237\n",
      "238\n",
      "239\n",
      "240\n",
      "241\n",
      "242\n",
      "243\n",
      "244\n",
      "245\n",
      "246\n",
      "247\n",
      "248\n",
      "249\n",
      "250\n",
      "251\n",
      "252\n",
      "253\n",
      "254\n",
      "255\n",
      "256\n",
      "257\n",
      "258\n",
      "259\n",
      "260\n",
      "261\n",
      "262\n",
      "263\n",
      "264\n",
      "265\n",
      "266\n",
      "267\n",
      "268\n",
      "269\n",
      "270\n",
      "271\n",
      "272\n",
      "273\n",
      "274\n",
      "275\n",
      "276\n",
      "277\n",
      "278\n",
      "279\n",
      "280\n",
      "281\n",
      "282\n",
      "283\n",
      "284\n",
      "285\n",
      "286\n",
      "287\n",
      "288\n",
      "289\n",
      "290\n",
      "291\n",
      "292\n",
      "293\n",
      "294\n",
      "295\n",
      "296\n",
      "297\n",
      "298\n",
      "299\n",
      "300\n",
      "301\n",
      "302\n",
      "303\n",
      "304\n",
      "305\n",
      "306\n",
      "307\n",
      "308\n",
      "309\n",
      "310\n",
      "311\n",
      "312\n",
      "313\n",
      "314\n",
      "315\n",
      "316\n",
      "317\n",
      "318\n",
      "319\n",
      "320\n",
      "321\n",
      "322\n",
      "323\n",
      "324\n",
      "325\n",
      "326\n",
      "327\n",
      "328\n",
      "329\n",
      "330\n",
      "331\n",
      "332\n",
      "333\n",
      "334\n",
      "335\n",
      "336\n",
      "337\n",
      "338\n",
      "339\n",
      "340\n",
      "341\n",
      "342\n",
      "343\n",
      "344\n",
      "345\n",
      "346\n",
      "347\n",
      "348\n",
      "349\n",
      "350\n",
      "351\n",
      "352\n",
      "353\n",
      "354\n",
      "355\n",
      "356\n",
      "357\n",
      "358\n",
      "359\n",
      "360\n",
      "361\n",
      "362\n",
      "363\n",
      "364\n",
      "365\n",
      "366\n",
      "367\n",
      "368\n",
      "369\n",
      "370\n",
      "371\n",
      "372\n",
      "373\n",
      "374\n",
      "375\n",
      "376\n",
      "377\n",
      "378\n",
      "379\n",
      "380\n",
      "381\n",
      "382\n",
      "383\n",
      "384\n",
      "385\n",
      "386\n",
      "387\n",
      "388\n",
      "389\n",
      "390\n",
      "391\n",
      "392\n",
      "393\n",
      "394\n",
      "395\n",
      "396\n",
      "397\n",
      "398\n",
      "399\n",
      "400\n",
      "401\n",
      "402\n",
      "403\n",
      "404\n",
      "405\n",
      "406\n",
      "407\n",
      "408\n",
      "409\n",
      "410\n",
      "411\n",
      "412\n",
      "413\n",
      "414\n",
      "415\n",
      "416\n",
      "417\n",
      "418\n",
      "419\n",
      "420\n",
      "421\n",
      "422\n",
      "423\n",
      "424\n",
      "425\n",
      "426\n",
      "427\n",
      "428\n",
      "429\n",
      "430\n",
      "431\n",
      "432\n",
      "433\n",
      "434\n",
      "435\n",
      "436\n",
      "437\n",
      "438\n",
      "439\n",
      "440\n",
      "441\n",
      "442\n",
      "443\n",
      "444\n",
      "445\n",
      "446\n",
      "447\n",
      "448\n",
      "449\n",
      "450\n",
      "451\n",
      "452\n",
      "453\n",
      "454\n",
      "455\n",
      "456\n",
      "457\n",
      "458\n",
      "459\n",
      "460\n",
      "461\n",
      "462\n",
      "463\n",
      "464\n",
      "465\n",
      "466\n",
      "467\n",
      "468\n",
      "469\n",
      "470\n",
      "471\n",
      "472\n",
      "473\n",
      "474\n",
      "475\n",
      "476\n",
      "477\n",
      "478\n",
      "479\n",
      "480\n",
      "481\n",
      "482\n",
      "483\n",
      "484\n",
      "485\n",
      "486\n",
      "487\n",
      "488\n",
      "489\n",
      "490\n",
      "491\n",
      "492\n",
      "493\n",
      "494\n",
      "495\n",
      "496\n",
      "497\n",
      "498\n",
      "499\n",
      "500\n",
      "501\n",
      "502\n",
      "503\n",
      "504\n",
      "505\n",
      "506\n",
      "507\n",
      "508\n",
      "509\n",
      "510\n",
      "511\n",
      "512\n",
      "513\n",
      "514\n",
      "515\n",
      "516\n",
      "517\n",
      "518\n",
      "519\n",
      "520\n",
      "521\n",
      "522\n",
      "523\n",
      "524\n",
      "525\n",
      "526\n",
      "527\n",
      "528\n",
      "529\n",
      "530\n",
      "531\n",
      "532\n",
      "533\n",
      "534\n",
      "535\n",
      "536\n",
      "537\n",
      "538\n",
      "539\n",
      "540\n",
      "541\n",
      "542\n",
      "543\n",
      "544\n",
      "545\n",
      "546\n",
      "547\n",
      "548\n",
      "549\n",
      "550\n",
      "551\n",
      "552\n",
      "553\n",
      "554\n",
      "555\n",
      "556\n",
      "557\n",
      "558\n",
      "559\n",
      "560\n",
      "561\n",
      "562\n",
      "563\n",
      "564\n",
      "565\n",
      "566\n",
      "567\n",
      "568\n",
      "569\n",
      "570\n",
      "571\n",
      "572\n",
      "573\n",
      "574\n",
      "575\n",
      "576\n",
      "577\n",
      "578\n",
      "579\n",
      "580\n",
      "581\n",
      "582\n",
      "583\n",
      "584\n",
      "585\n",
      "586\n",
      "587\n",
      "588\n",
      "589\n",
      "590\n",
      "591\n",
      "592\n",
      "593\n",
      "594\n",
      "595\n",
      "596\n",
      "597\n",
      "598\n",
      "599\n",
      "600\n",
      "601\n",
      "602\n",
      "603\n",
      "604\n",
      "605\n",
      "606\n",
      "607\n",
      "608\n",
      "609\n",
      "610\n",
      "611\n",
      "612\n",
      "613\n",
      "614\n",
      "615\n",
      "616\n",
      "617\n",
      "618\n",
      "619\n",
      "620\n",
      "621\n",
      "622\n",
      "623\n",
      "624\n",
      "625\n",
      "626\n",
      "627\n",
      "628\n",
      "629\n",
      "630\n",
      "631\n",
      "632\n",
      "633\n",
      "634\n",
      "635\n",
      "636\n",
      "637\n",
      "638\n",
      "639\n",
      "640\n",
      "641\n",
      "642\n",
      "643\n",
      "644\n",
      "645\n",
      "646\n",
      "647\n",
      "648\n",
      "649\n",
      "650\n",
      "651\n",
      "652\n",
      "653\n",
      "654\n",
      "655\n",
      "656\n",
      "657\n",
      "658\n",
      "659\n",
      "660\n",
      "661\n",
      "662\n",
      "663\n",
      "664\n",
      "665\n",
      "666\n",
      "667\n",
      "668\n",
      "669\n",
      "670\n",
      "671\n",
      "672\n",
      "673\n",
      "674\n",
      "675\n",
      "676\n",
      "677\n",
      "678\n",
      "679\n",
      "680\n",
      "681\n",
      "682\n",
      "683\n",
      "684\n",
      "685\n",
      "686\n",
      "687\n",
      "688\n",
      "689\n",
      "690\n",
      "691\n",
      "692\n",
      "693\n",
      "694\n",
      "695\n",
      "696\n",
      "697\n",
      "698\n",
      "699\n",
      "700\n",
      "701\n",
      "702\n",
      "703\n",
      "704\n",
      "705\n",
      "706\n",
      "707\n",
      "708\n",
      "709\n",
      "710\n",
      "711\n",
      "712\n",
      "713\n",
      "714\n",
      "715\n",
      "716\n",
      "717\n",
      "718\n",
      "719\n",
      "720\n",
      "721\n",
      "722\n",
      "723\n",
      "724\n",
      "725\n",
      "726\n",
      "727\n",
      "728\n",
      "729\n",
      "730\n",
      "731\n",
      "732\n",
      "733\n",
      "734\n",
      "735\n",
      "736\n",
      "737\n",
      "738\n",
      "739\n",
      "740\n",
      "741\n",
      "742\n",
      "743\n",
      "744\n",
      "745\n",
      "746\n",
      "747\n",
      "748\n",
      "749\n",
      "750\n",
      "751\n",
      "752\n",
      "753\n",
      "754\n",
      "755\n",
      "756\n",
      "757\n",
      "758\n",
      "759\n",
      "760\n",
      "761\n",
      "762\n",
      "763\n",
      "764\n",
      "765\n",
      "766\n",
      "767\n",
      "768\n",
      "769\n",
      "770\n",
      "771\n",
      "772\n",
      "773\n",
      "774\n",
      "775\n",
      "776\n",
      "777\n",
      "778\n",
      "779\n",
      "780\n",
      "781\n",
      "782\n",
      "783\n",
      "784\n",
      "785\n",
      "786\n",
      "787\n",
      "788\n",
      "789\n",
      "790\n",
      "791\n",
      "792\n",
      "793\n",
      "794\n",
      "795\n",
      "796\n",
      "797\n",
      "798\n",
      "799\n",
      "800\n",
      "801\n",
      "802\n",
      "803\n",
      "804\n",
      "805\n",
      "806\n",
      "807\n",
      "808\n",
      "809\n",
      "810\n",
      "811\n",
      "812\n",
      "813\n",
      "814\n",
      "815\n",
      "816\n",
      "817\n",
      "818\n",
      "819\n",
      "820\n",
      "821\n",
      "822\n",
      "823\n",
      "824\n",
      "825\n",
      "826\n",
      "827\n",
      "828\n",
      "829\n",
      "830\n",
      "831\n",
      "832\n",
      "833\n",
      "834\n",
      "835\n",
      "836\n",
      "837\n",
      "838\n",
      "839\n",
      "840\n",
      "841\n",
      "842\n",
      "843\n",
      "844\n",
      "845\n",
      "846\n",
      "847\n",
      "848\n",
      "849\n",
      "850\n",
      "851\n",
      "852\n",
      "853\n",
      "854\n",
      "855\n",
      "856\n",
      "857\n",
      "858\n",
      "859\n",
      "860\n",
      "861\n",
      "862\n",
      "863\n",
      "864\n",
      "865\n",
      "866\n",
      "867\n",
      "868\n",
      "869\n",
      "870\n",
      "871\n",
      "872\n",
      "873\n",
      "874\n",
      "875\n",
      "876\n",
      "877\n",
      "878\n",
      "879\n",
      "880\n",
      "881\n",
      "882\n",
      "883\n",
      "884\n",
      "885\n",
      "886\n",
      "887\n",
      "888\n",
      "889\n",
      "890\n",
      "891\n",
      "892\n",
      "893\n",
      "894\n",
      "895\n",
      "896\n",
      "897\n",
      "898\n",
      "899\n",
      "900\n",
      "901\n",
      "902\n",
      "903\n",
      "904\n",
      "905\n",
      "906\n",
      "907\n",
      "908\n",
      "909\n",
      "910\n",
      "911\n",
      "912\n",
      "913\n",
      "914\n",
      "915\n",
      "916\n",
      "917\n",
      "918\n",
      "919\n",
      "920\n",
      "921\n",
      "922\n",
      "923\n",
      "924\n",
      "925\n",
      "926\n",
      "927\n",
      "928\n",
      "929\n",
      "930\n",
      "931\n",
      "932\n",
      "933\n",
      "934\n",
      "935\n",
      "936\n",
      "937\n",
      "938\n",
      "939\n",
      "940\n",
      "941\n",
      "942\n",
      "943\n",
      "944\n",
      "945\n",
      "946\n",
      "947\n",
      "948\n",
      "949\n",
      "950\n",
      "951\n",
      "952\n",
      "953\n",
      "954\n",
      "955\n",
      "956\n",
      "957\n",
      "958\n",
      "959\n",
      "960\n",
      "961\n",
      "962\n",
      "963\n",
      "964\n",
      "965\n",
      "966\n",
      "967\n",
      "968\n",
      "969\n",
      "970\n",
      "971\n",
      "972\n",
      "973\n",
      "974\n",
      "975\n",
      "976\n",
      "977\n",
      "978\n",
      "979\n",
      "980\n",
      "981\n",
      "982\n",
      "983\n",
      "984\n",
      "985\n",
      "986\n",
      "987\n",
      "988\n",
      "989\n",
      "990\n",
      "991\n",
      "992\n",
      "993\n",
      "994\n",
      "995\n",
      "996\n",
      "997\n",
      "998\n",
      "999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 33/33 [02:53<00:00,  5.24s/it]\n",
      "100%|██████████| 253378/253378 [02:14<00:00, 1887.85it/s]\n"
     ]
    }
   ],
   "source": [
    "%run ../twitter16/userdata_parser.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 253378/253378 [00:01<00:00, 192397.89it/s]\n"
     ]
    }
   ],
   "source": [
    "for key in tqdm(userVects):\n",
    "    userVects[key] = userVects[key].float()\n",
    "\n",
    "userVects = defaultdict(lambda:torch.tensor([1.1100e+02, 1.5000e+01, 0.0000e+00, 7.9700e+02, 4.7300e+02, 0.0000e+00,\n",
    "        8.3326e+04, 1.0000e+00]),userVects)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading All Architectures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%run ./temporal_tree_model.ipynb "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'false': 0, 'true': 1, 'unverified': 2, 'non-rumor': 3}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labelMap = {}\n",
    "labelCount = 0\n",
    "for label in list(twitter15_labels.values()):\n",
    "    if label not in labelMap:\n",
    "        labelMap[label] = labelCount\n",
    "        labelCount += 1\n",
    "labelMap"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optim and Loss Fxn & Creating Model Inst of Regular Temporal Tree Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = torch.nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = treeEncoder(torch.cuda.is_available(),8,100,userVects,twitter15_labels,labelMap,criterion,device)\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = ChildSumTreeLSTM(torch.cuda.is_available(),8,30,userVects,twitter15_labels,labelMap,criterion,device)\n",
    "# model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model(twitter15_trees[436146437530075136][1].root,userVects)\n",
    "# make_dot(model(twitter15_trees[436146437530075136][1].root,userVects)[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# checkpoint = torch.load('./tempTreeEnc.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model = lstmTreeEncoder(torch.cuda.is_available(),8,100,userVects,twitter15_labels,labelMap,criterion,device)\n",
    "model = model.to(device)\n",
    "# model.load_state_dict(checkpoint['state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adagrad(model.parameters(),lr=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# sample_pred = model(twitter15_trees[537913349338435584][1].root)\n",
    "# sample_pred[1]\n",
    "# x = make_dot(model(twitter15_trees[537913349338435584][1].root)[0][1], params=dict(model.named_parameters()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x.render('./test.png',format='png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(userVects.keys())[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "userVects[17377547]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sample_pred = model(twitter15_trees[537913349338435584][3].root)[1]\n",
    "sample_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getBack(var_grad_fn):\n",
    "    print(var_grad_fn)\n",
    "    for n in var_grad_fn.next_functions:\n",
    "        if n[0]:\n",
    "            try:\n",
    "                tensor = getattr(n[0], 'variable')\n",
    "                print(n[0])\n",
    "                print('Tensor with grad found:', tensor)\n",
    "                print(' - gradient:', tensor.grad)\n",
    "                print()\n",
    "            except AttributeError as e:\n",
    "                getBack(n[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dict(model.named_parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "make_dot(sample_pred, params=dict(model.named_parameters()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "y = []\n",
    "for tid in twitter15_trees:\n",
    "        if tid in twitter15_trees and tid in twitter15_labels:\n",
    "            X.append(twitter15_trees[tid])\n",
    "            y.append(twitter15_labels[tid])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test = train_test_split(X,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trainer for Tree Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "613"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = torch.load('./treeEncOpt.pth')\n",
    "model = treeEncoder(torch.cuda.is_available(),8,100,userVects,twitter15_labels,labelMap,criterion,device)\n",
    "model.load_state_dict(checkpoint['state_dict'])\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters(),lr=0.01)\n",
    "\n",
    "count = 0\n",
    "netloss = 0\n",
    "\n",
    "train_iterwise = []\n",
    "val_iterwise = []\n",
    "\n",
    "for i in range(epochs):\n",
    "    train_losses = []\n",
    "    val_losses = []\n",
    "    \n",
    "    for treeSet in tqdm_notebook(x_train):\n",
    "            tnum = 0\n",
    "            tree = treeSet[-1]\n",
    "#         for tree in treeSet:\n",
    "#             print(count)\n",
    "            count += 1\n",
    "            tnum += 1\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            (h,c),loss = model(tree.root)\n",
    "        \n",
    "            label = Variable(torch.tensor(labelMap[tree.root.label]))\n",
    "            \n",
    "            if torch.cuda.is_available():\n",
    "                label.to(device)\n",
    "#             print(loss)\n",
    "            netloss += loss\n",
    "    \n",
    "            if count % 10 == 0:\n",
    "#                 print('opt')\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "            \n",
    "    preds = []\n",
    "    labels = []\n",
    "\n",
    "    allLabels = []\n",
    "    allPreds = []\n",
    "\n",
    "    for valSet in x_test:\n",
    "        finalTree = valSet[-1]\n",
    "        predicted = model.predict(finalTree.root)\n",
    "        preds.append(predicted)\n",
    "        print(predicted)\n",
    "\n",
    "        predicted =  torch.softmax(predicted[0],0)\n",
    "        predicted = torch.max(predicted, 0)[1].cpu().numpy().tolist()\n",
    "\n",
    "        labels.append(labelMap[finalTree.root.label])\n",
    "\n",
    "        allLabels.append(labelMap[finalTree.root.label])\n",
    "        allPreds.append(predicted)\n",
    "\n",
    "    predTensor = torch.stack(preds)\n",
    "    labelTensor = torch.tensor(labels).to(device)\n",
    "\n",
    "    print(allLabels,allPreds)\n",
    "\n",
    "    loss = criterion(predTensor.reshape(-1,4), labelTensor.reshape(-1))\n",
    "\n",
    "    cr = classification_report(allLabels,allPreds,output_dict=True)\n",
    "    cr['loss'] = loss.item()\n",
    "    cr['Acc'] = accuracy_score(allLabels,allPreds,)\n",
    "    print('loss: ',cr['loss'])\n",
    "    print(cr['Acc'])\n",
    "#     with open('treeEnc.json', 'a') as fp:\n",
    "#         json.dump(cr, fp)\n",
    "#         fp.write('\\n')\n",
    "    #                     print(predicted)\n",
    "    val_losses.append(loss.item())\n",
    "    train_iterwise.append(np.array(train_losses).mean())\n",
    "    val_iterwise.append(np.array(val_losses).mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trainer for Tree + Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run ./temporal_tree_model.ipynb "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('../baselines/sentence2idx_twit16.pkl','rb') as f:\n",
    "    sent2idx = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = torch.nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = decayTreeText(torch.cuda.is_available(),8,100,userVects,twitter15_labels,labelMap,criterion,device)\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a147dbe33302474ba084577db00b3508",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nikhil.pinnaparaju/anaconda3/envs/fakenews/lib/python3.7/site-packages/ipykernel_launcher.py:17: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6982f91384614e3b94fb8b4136999df5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 3, 1, 1, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 1, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 3, 3, 3, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 1, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 0, 1, 2, 3, 3, 3, 1, 3, 3, 3, 0, 3, 2, 0, 0, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 3, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.667400062084198\n",
      "0.9365853658536586\n",
      "1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aff9e543f2254534929447b08d8103c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c6d345255af425fb5293308185dd4fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 2, 0, 1, 3, 2, 1, 1, 2, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 2, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 0, 3, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 0, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 2, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 2, 0, 0, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 2, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.49104607105255127\n",
      "0.9365853658536586\n",
      "2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "679c102a55d949fea5d21575246111c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "933f73faef3d4d669dd866df4b3c29e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 3, 0, 3, 3, 1, 3, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 3, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 3, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 3, 3, 3, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 1, 1, 2, 2, 1, 1, 3, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 1, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 3, 1, 2, 3, 3, 3, 1, 3, 3, 3, 0, 3, 3, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 3, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.44672104716300964\n",
      "0.9219512195121952\n",
      "3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "babd7703c2b44c9eba8d13568589829c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d0d933f7a1c43fe8993d8c87b1fb382",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 3, 1, 1, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 3, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 3, 3, 3, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 1, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 1, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 3, 1, 2, 3, 3, 3, 1, 3, 3, 3, 0, 3, 2, 0, 0, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 3, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.36907416582107544\n",
      "0.9317073170731708\n",
      "4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "044034fd3c3b4e7180f05a11180b9cc4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b9944acd8dd4598a08861e1038f0bd9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 3, 3, 3, 0, 2, 1, 3, 0, 1, 1, 3, 3, 0, 0, 3, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 3, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 3, 3, 3, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 3, 1, 2, 2, 1, 1, 3, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 0, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 3, 1, 2, 3, 3, 3, 1, 3, 3, 3, 0, 3, 2, 0, 0, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 3, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.3596785366535187\n",
      "0.9073170731707317\n",
      "5\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9901024ff3de4489b14c8c018b6f5669",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91eb253f488f40abb5370a4fe7d6c235",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 3, 1, 1, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 3, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 3, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 2, 3, 3, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 1, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 3, 1, 0, 3, 2, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 2, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.2841040790081024\n",
      "0.9414634146341463\n",
      "6\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3e092fa8ea394d3fbc35a3920d71d514",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8402cd8c769c42b08611e11e6bab63b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 3, 1, 3, 0, 2, 1, 3, 0, 1, 1, 1, 3, 0, 0, 3, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 3, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 3, 3, 3, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 1, 1, 2, 2, 1, 1, 3, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 3, 1, 2, 3, 3, 3, 1, 3, 3, 3, 0, 3, 2, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 3, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.28036123514175415\n",
      "0.926829268292683\n",
      "7\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a89ba4ac0f3491ab489953512a21632",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "582b48657033499b87c6400b780d69fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 3, 1, 3, 0, 2, 1, 3, 0, 1, 1, 1, 3, 0, 0, 3, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 3, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 3, 3, 3, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 1, 1, 2, 2, 1, 1, 3, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 3, 1, 2, 3, 3, 3, 1, 3, 3, 3, 0, 3, 2, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 3, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.2671475112438202\n",
      "0.926829268292683\n",
      "8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45de1bf1c15548408479146b23d9a549",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb88f3e398914371b1a5033a9a722750",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 3, 1, 3, 0, 2, 1, 3, 0, 1, 1, 1, 3, 0, 0, 3, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 3, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 3, 3, 3, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 1, 1, 2, 2, 1, 1, 3, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 3, 1, 2, 3, 3, 3, 1, 3, 3, 3, 0, 3, 2, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 3, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.26256459951400757\n",
      "0.926829268292683\n",
      "9\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28a19b5db7f342e38f4e788b615d9286",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "290c1a11c5aa4162974913b90d281a87",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 3, 1, 3, 0, 2, 1, 3, 0, 1, 1, 1, 3, 0, 0, 3, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 3, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 3, 3, 3, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 1, 1, 2, 2, 1, 1, 3, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 3, 1, 2, 3, 3, 3, 1, 3, 3, 3, 0, 3, 2, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 3, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.2479834258556366\n",
      "0.926829268292683\n"
     ]
    }
   ],
   "source": [
    "optimizer = torch.optim.Adagrad(model.parameters(),0.01)\n",
    "\n",
    "maxAcc = 0\n",
    "count = 0\n",
    "netloss = 0\n",
    "\n",
    "for i in range(10):\n",
    "    print(i)\n",
    "    train_losses = []\n",
    "    val_losses = []\n",
    "    \n",
    "    for treeSet in tqdm_notebook(x_train):\n",
    "            tnum = 0\n",
    "            tree = treeSet[-1]\n",
    "#         for tree in treeSet:\n",
    "#             print(count)\n",
    "            count += 1\n",
    "#             tnum += 1\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            text = torch.tensor(sent2idx[tree.tweet_id])\n",
    "            text = Variable(text.view(-1, len(text), 1)).to(device)\n",
    "            \n",
    "            pred = model(tree.root,text)\n",
    "            \n",
    "            label = Variable(torch.tensor(labelMap[treeSet[0].root.label]).reshape(-1).to(device))\n",
    "            loss = criterion(pred.reshape(1,4),label)\n",
    "#             print(loss)\n",
    "            netloss += loss\n",
    "    \n",
    "            if count % 20 == 0:\n",
    "#                 print('opt')\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "            \n",
    "    preds = []\n",
    "    labels = []\n",
    "\n",
    "    allLabels = []\n",
    "    allPreds = []\n",
    "\n",
    "    for valSet in tqdm_notebook(x_test):\n",
    "        finalTree = valSet[-1]\n",
    "        \n",
    "        text = torch.tensor(sent2idx[finalTree.tweet_id])\n",
    "        text = Variable(text.view(-1, len(text), 1)).to(device)\n",
    "        \n",
    "        predicted = model(finalTree.root,text)\n",
    "        preds.append(predicted)\n",
    "#         print(predicted)\n",
    "        predicted =  torch.softmax(predicted,0)\n",
    "        predicted = torch.max(predicted, 0)[1].cpu().numpy().tolist()\n",
    "\n",
    "        labels.append(labelMap[finalTree.root.label])\n",
    "\n",
    "        allLabels.append(labelMap[finalTree.root.label])\n",
    "        allPreds.append(predicted)\n",
    "\n",
    "    predTensor = torch.stack(preds)\n",
    "    labelTensor = torch.tensor(labels).to(device)\n",
    "\n",
    "    print(allLabels,allPreds)\n",
    "\n",
    "    loss = criterion(predTensor.reshape(-1,4), labelTensor.reshape(-1))\n",
    "\n",
    "    cr = classification_report(allLabels,allPreds,output_dict=True)\n",
    "    cr['loss'] = loss.item()\n",
    "    cr['Acc'] = accuracy_score(allLabels,allPreds,)\n",
    "    \n",
    "    if cr['Acc'] > maxAcc:\n",
    "        maxAcc = cr['Acc']\n",
    "        torch.save({'state_dict': model.state_dict()}, './decayTreeText_twit16.pth')\n",
    "    \n",
    "    print('loss: ',cr['loss'])\n",
    "    print(cr['Acc'])\n",
    "    \n",
    "    with open('decayTreeText_twit16.json', 'a') as fp:\n",
    "        json.dump(cr, fp)\n",
    "        fp.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "device='cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = treeText(torch.cuda.is_available(),8,100,userVects,twitter15_labels,labelMap,criterion,device)\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checkpoint = torch.load('./textTree_twit16.pth')\n",
    "model.load_state_dict(checkpoint['state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3af1a041243c4e0b832a1ec9c02fff6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nikhil.pinnaparaju/anaconda3/envs/fakenews/lib/python3.7/site-packages/ipykernel_launcher.py:17: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "749b7786499a4c6baaaa7f48e44709f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 3, 1, 3, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 3, 0, 1, 3, 3, 1, 1, 1, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 1, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 1, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 1, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 3, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 1, 3, 3, 2, 1, 2, 3, 2, 1, 3, 2, 3]\n",
      "loss:  0.5637508034706116\n",
      "0.9512195121951219\n",
      "1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa4de7daf1a947b3b1e8e5d639866d7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "75449e83bf6a4eb1aca0d05db008e89a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 2, 0, 1, 3, 2, 1, 1, 2, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 2, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 2, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 2, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 2, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.4018317759037018\n",
      "0.9560975609756097\n",
      "2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e31e17b9ffc142218df189e389c3b302",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94b92288d08247669c4e332770dc4c37",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 0, 1, 2, 3, 0, 0, 3, 3, 1, 3, 0, 2, 1, 3, 0, 1, 1, 3, 3, 0, 0, 3, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 3, 3, 3, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 3, 1, 2, 2, 1, 1, 3, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 3, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 3, 1, 2, 3, 3, 3, 1, 3, 3, 3, 0, 3, 3, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 3, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.36996322870254517\n",
      "0.9170731707317074\n",
      "3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85298663bc984dc490fae05c09400698",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4efedb7625364421b283e49e4abf4701",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 0, 1, 2, 3, 0, 0, 3, 3, 1, 3, 0, 2, 1, 3, 0, 1, 1, 1, 3, 0, 0, 3, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 3, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 3, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 3, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 3, 1, 2, 3, 3, 3, 1, 3, 3, 3, 0, 3, 3, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.28855907917022705\n",
      "0.9365853658536586\n",
      "4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9649892d85a54fb18b3a3c9d9308a90f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87ccdb522f90421aab721f923294471d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 0, 1, 2, 3, 0, 0, 3, 3, 3, 2, 0, 2, 1, 3, 0, 1, 1, 3, 3, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 3, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 3, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 3, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 0, 3, 0, 3, 2, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.28442370891571045\n",
      "0.9414634146341463\n",
      "5\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8aa48672af894689a0d78103bb55d7e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03a2606fc2374a11b96730295026f3ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 0, 1, 2, 3, 0, 0, 3, 0, 3, 2, 0, 2, 1, 3, 0, 1, 1, 1, 3, 0, 0, 0, 0, 1, 3, 0, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 3, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 3, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 3, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 0, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 0, 3, 0, 3, 2, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 0, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.26201650500297546\n",
      "0.9317073170731708\n",
      "6\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a6374b663014df4ae9d91226c1330b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "156b4b7aa659478189301cce6ff45871",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 0, 1, 2, 3, 0, 0, 3, 0, 3, 3, 0, 2, 1, 3, 0, 1, 1, 3, 3, 0, 0, 0, 0, 1, 3, 0, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 3, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 3, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 3, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 0, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 0, 3, 0, 3, 2, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.2554987370967865\n",
      "0.926829268292683\n",
      "7\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fab41176ebf34ea0814ebd78f6fbaa8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68377ae83e6845b68d34bbe22c8f6f78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 0, 1, 2, 3, 0, 0, 3, 0, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 3, 0, 0, 0, 0, 1, 3, 0, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 3, 2, 0, 2, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 0, 3, 0, 3, 2, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 0, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.21221806108951569\n",
      "0.9463414634146341\n",
      "8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "20a625287a8540328f94098af3947f89",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71a5617734cc453c9f479df093a78e57",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 0, 1, 2, 3, 0, 0, 3, 3, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 3, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 3, 2, 0, 2, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 0, 3, 0, 3, 2, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.19123707711696625\n",
      "0.9560975609756097\n",
      "9\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23e4c46e33694242b4402d56a44724aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=613), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8687589d0874bc587d3e50aae55ba90",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 2, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 1, 2, 0, 0, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 2, 3, 0, 3, 1, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 1, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 0, 3, 2, 3] [0, 3, 0, 1, 3, 2, 3, 2, 1, 2, 3, 0, 0, 3, 3, 1, 2, 0, 2, 1, 3, 0, 1, 1, 1, 3, 0, 0, 0, 0, 1, 3, 3, 1, 1, 3, 1, 1, 2, 0, 0, 0, 1, 3, 1, 3, 1, 2, 3, 2, 1, 3, 3, 2, 2, 0, 1, 3, 2, 0, 0, 1, 0, 3, 1, 2, 2, 3, 1, 0, 3, 3, 1, 3, 3, 2, 1, 2, 2, 1, 1, 3, 2, 0, 2, 3, 1, 1, 3, 1, 1, 2, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 3, 0, 2, 1, 3, 0, 2, 3, 3, 1, 3, 1, 3, 0, 2, 2, 1, 1, 0, 1, 2, 1, 2, 3, 2, 2, 0, 0, 0, 2, 0, 3, 0, 2, 3, 2, 1, 2, 1, 2, 3, 3, 3, 1, 3, 0, 3, 0, 3, 2, 0, 3, 2, 2, 2, 3, 0, 2, 3, 0, 1, 2, 0, 1, 0, 2, 2, 1, 1, 2, 3, 2, 3, 2, 3, 3, 0, 1, 2, 3, 3, 3, 3, 2, 2, 1, 3, 1, 0, 0, 0, 0, 3, 3, 2, 1, 2, 3, 2, 3, 3, 2, 3]\n",
      "loss:  0.17622438073158264\n",
      "0.9609756097560975\n"
     ]
    }
   ],
   "source": [
    "optimizer = torch.optim.Adagrad(model.parameters(),0.01)\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "\n",
    "count = 0\n",
    "netloss = 0\n",
    "\n",
    "maxAcc = 0\n",
    "\n",
    "for i in range(10):\n",
    "    print(i)\n",
    "    train_losses = []\n",
    "    val_losses = []\n",
    "    \n",
    "    for treeSet in tqdm_notebook(x_train):\n",
    "            tnum = 0\n",
    "            tree = treeSet[-1]\n",
    "#         for tree in treeSet:\n",
    "#             print(count)\n",
    "            count += 1\n",
    "#             tnum += 1\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            text = torch.tensor(sent2idx[tree.tweet_id])\n",
    "            text = Variable(text.view(-1, len(text), 1)).to(device)\n",
    "            \n",
    "            pred = model(tree.root,text)\n",
    "            \n",
    "            label = Variable(torch.tensor(labelMap[treeSet[0].root.label]).reshape(-1).to(device))\n",
    "            loss = criterion(pred.reshape(1,4),label)\n",
    "#             print(loss)\n",
    "            netloss += loss\n",
    "    \n",
    "            if count % 20 == 0:\n",
    "#                 print('opt')\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "            \n",
    "    preds = []\n",
    "    labels = []\n",
    "\n",
    "    allLabels = []\n",
    "    allPreds = []\n",
    "\n",
    "    for valSet in tqdm_notebook(x_test):\n",
    "        finalTree = valSet[-1]\n",
    "        \n",
    "        text = torch.tensor(sent2idx[finalTree.tweet_id])\n",
    "        text = Variable(text.view(-1, len(text), 1)).to(device)\n",
    "        \n",
    "        predicted = model(finalTree.root,text)\n",
    "        preds.append(predicted)\n",
    "#         print(predicted)\n",
    "        predicted =  torch.softmax(predicted,0)\n",
    "        predicted = torch.max(predicted, 0)[1].cpu().numpy().tolist()\n",
    "\n",
    "        labels.append(labelMap[finalTree.root.label])\n",
    "\n",
    "        allLabels.append(labelMap[finalTree.root.label])\n",
    "        allPreds.append(predicted)\n",
    "\n",
    "    predTensor = torch.stack(preds)\n",
    "    labelTensor = torch.tensor(labels).to(device)\n",
    "\n",
    "    print(allLabels,allPreds)\n",
    "\n",
    "    loss = criterion(predTensor.reshape(-1,4), labelTensor.reshape(-1))\n",
    "\n",
    "    cr = classification_report(allLabels,allPreds,output_dict=True)\n",
    "    cr['loss'] = loss.item()\n",
    "    cr['Acc'] = accuracy_score(allLabels,allPreds,)\n",
    "    print('loss: ',cr['loss'])\n",
    "    print(cr['Acc'])\n",
    "    \n",
    "    if cr['Acc'] > maxAcc:\n",
    "        maxAcc = cr['Acc']\n",
    "        torch.save({'state_dict': model.state_dict()}, './treeText_twit16.pth')\n",
    "    \n",
    "    with open('treeText_twit16.json', 'a') as fp:\n",
    "        json.dump(cr, fp)\n",
    "        fp.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  torch.save({'state_dict': model.state_dict()}, './treeText.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = torch.load('./treeText.pth')\n",
    "model.load_state_dict(checkpoint['state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = []\n",
    "labels = []\n",
    "\n",
    "allLabels = []\n",
    "allPreds = []\n",
    "    \n",
    "for valSet in tqdm_notebook(x_test):\n",
    "    finalTree = valSet[-1]\n",
    "\n",
    "    text = torch.tensor(sent2idx[finalTree.tweet_id])\n",
    "    text = Variable(text.view(-1, len(text), 1)).to(device)\n",
    "\n",
    "    predicted = model(finalTree.root,text)\n",
    "    preds.append(predicted)\n",
    "#         print(predicted)\n",
    "    predicted =  torch.softmax(predicted,0)\n",
    "    predicted = torch.max(predicted, 0)[1].cpu().numpy().tolist()\n",
    "\n",
    "    labels.append(labelMap[finalTree.root.label])\n",
    "\n",
    "    allLabels.append(labelMap[finalTree.root.label])\n",
    "    allPreds.append(predicted)\n",
    "\n",
    "predTensor = torch.stack(preds)\n",
    "labelTensor = torch.tensor(labels).to(device)\n",
    "\n",
    "print(allLabels,allPreds)\n",
    "\n",
    "loss = criterion(predTensor.reshape(-1,4), labelTensor.reshape(-1))\n",
    "\n",
    "cr = classification_report(allLabels,allPreds,output_dict=True)\n",
    "cr['loss'] = loss.item()\n",
    "cr['Acc'] = accuracy_score(allLabels,allPreds,)\n",
    "print('loss: ',cr['loss'])\n",
    "print(cr['Acc'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trainer for Temporal Tree Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lossfile = './lossesTempEnc.txt'\n",
    "f = open(lossfile, \"a\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "count = 0\n",
    "for i in range(epochs):    \n",
    "    for treeSet in tqdm_notebook(x_train):\n",
    "        count += 1\n",
    "        optimizer.zero_grad()\n",
    "            \n",
    "        pred = model(treeSet)\n",
    "        \n",
    "        label = Variable(torch.tensor(labelMap[treeSet[0].root.label]).reshape(-1).to(device))\n",
    "        loss = criterion(pred,label)\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "            \n",
    "    preds = []\n",
    "\n",
    "    for i in range(len(x_test)):\n",
    "        valTreeSet = x_test[i]\n",
    "        preds.append(model(valTreeSet))\n",
    "\n",
    "    predTensor = torch.stack(preds)\n",
    "    labelTensor = torch.tensor([labelMap[i] for i in y_test]).to(device)\n",
    "\n",
    "#     print(predTensor)\n",
    "#     print(labelTensor)\n",
    "\n",
    "    loss = criterion(predTensor.reshape(-1,4), labelTensor.reshape(-1))\n",
    "    print(loss)\n",
    "    f.write(str(loss.item()))\n",
    "                \n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "count = 0\n",
    "for i in range(epochs):    \n",
    "    for treeSet in tqdm_notebook(x_train):     \n",
    "        count += 1\n",
    "        optimizer.zero_grad()\n",
    "            \n",
    "        pred = model(treeSet).reshape(1,-1)\n",
    "        \n",
    "        label = Variable(torch.tensor(labelMap[treeSet[0].root.label]).reshape(-1).to(device))\n",
    "        print(pred,label)\n",
    "        loss = criterion(torch.tensor([[1,0,0,0]]).float().requires_grad_(),label)\n",
    "                \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if count % 50 == 0:\n",
    "            torch.save({'state_dict': model.state_dict()}, './tempTreeEnc.pth')\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(model,'./tempTreeEnc.pth')\n",
    "torch.save({'state_dict': model.state_dict()}, './tempTreeEnc.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "preds = []\n",
    "            \n",
    "for i in range(len(x_test)):\n",
    "    valTreeSet = x_test[i]\n",
    "    preds.append(model(valTreeSet).detach())\n",
    "                \n",
    "    predTensor = torch.stack(preds)\n",
    "    labelTensor = torch.tensor([labelMap[i] for i in y_test]).to(device)\n",
    "    loss = criterion(predTensor.reshape(-1,4), labelTensor.reshape(-1))\n",
    "    print('Loss Value: ', loss.detach().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "preds = []\n",
    "labels = []\n",
    "\n",
    "for valSet in x_test:\n",
    "    finalTree = valSet[-1]\n",
    "    preds.append(model.treeEnc(finalTree.root)[1].detach())\n",
    "#     labels.append(labelMap[finalTree.root.label])\n",
    "    print(preds)\n",
    "    predTensor = torch.stack(preds)\n",
    "#     labelTensor = torch.tensor(labels).to(device)\n",
    "\n",
    "#     loss = criterion(predTensor.reshape(-1,4), labelTensor.reshape(-1))\n",
    "#     print('Loss Value: ', loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "preds = []\n",
    "labels = []\n",
    "\n",
    "for valSet in x_test:\n",
    "    finalTree = valSet[-1]\n",
    "    preds.append(model.treeEnc.predict(finalTree.root))\n",
    "    labels.append(labelMap[finalTree.root.label])\n",
    "                \n",
    "    predTensor = torch.stack(preds)\n",
    "    labelTensor = torch.tensor(labels).to(device)\n",
    "#print(predTensor)\n",
    "#print(labelTensor)\n",
    "    loss = criterion(predTensor.reshape(-1,4), labelTensor.reshape(-1))\n",
    "    print('Loss Value: ', loss.item())\n",
    "#     val_losses.append(loss.item())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plotting Losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_iterwise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iterNums = [i for i in range(len(train_iterwise))]\n",
    "sns.lineplot(iterNums,train_iterwise)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iterNums = [i for i in range(len(val_losses))]\n",
    "sns.lineplot(iterNums,val_losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(train_losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lenAggreg = 0\n",
    "for subset in x_train:\n",
    "    lenAggreg += len(subset)\n",
    "print(lenAggreg)\n",
    "print(lenAggreg/len(x_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training Temporal Decay Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = temporalDecayTreeEncoder(cuda,8,30,userVects,labels,labelMap,criterion,device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "testModel(x_train[0][0].root)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = torch.load('./tempDecayTreeEnc.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = temporalDecayTreeEncoder(torch.cuda.is_available(),8,30,userVects,twitter15_labels,labelMap,criterion,device)\n",
    "model = model.to(device)\n",
    "model.load_state_dict(checkpoint['state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters(),lr = 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lossfile = './lossesDecayTempEnc.txt'\n",
    "f = open(lossfile, \"a\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 0\n",
    "for i in range(epochs):    \n",
    "    for treeSet in tqdm_notebook(x_train):     \n",
    "        count += 1\n",
    "        optimizer.zero_grad()\n",
    "            \n",
    "        pred = model(treeSet)\n",
    "        \n",
    "        label = Variable(torch.tensor(labelMap[treeSet[0].root.label]).reshape(-1).to(device))\n",
    "        loss = criterion(pred,label)\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "            \n",
    "        if count % 500 == 0:\n",
    "            preds = []\n",
    "            \n",
    "            for i in range(len(x_test)):\n",
    "                valTreeSet = x_test[i]\n",
    "                preds.append(model(valTreeSet))\n",
    "                \n",
    "                predTensor = torch.stack(preds)\n",
    "                labelTensor = torch.tensor([labelMap[i] for i in y_test]).to(device)\n",
    "                loss = criterion(predTensor.reshape(-1,4), labelTensor.reshape(-1))\n",
    "                \n",
    "                f.write(str(loss.item()))\n",
    "                \n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(model,'./tempTreeEnc.pth')\n",
    "torch.save({'state_dict': model.state_dict()}, './tempDecayTreeEnc.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = []\n",
    "labels = []\n",
    "\n",
    "for valSet in x_test:\n",
    "    finalTree = valSet[-1]\n",
    "    preds.append(model.treeEnc.predict(finalTree.root))\n",
    "    labels.append(labelMap[finalTree.root.label])\n",
    "                \n",
    "    predTensor = torch.stack(preds)\n",
    "    labelTensor = torch.tensor(labels).to(device)\n",
    "#print(predTensor)\n",
    "#print(labelTensor)\n",
    "    loss = criterion(predTensor.reshape(-1,4), labelTensor.reshape(-1))\n",
    "    print('Loss Value: ', loss.item())\n",
    "    val_losses.append(loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labelMap[y_test[1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sampleout = model(x_test[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampleout[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampleout[0].max(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(model.modules())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "se = SizeEstimator(model, input_size=(16,1,256,256))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "se.get_parameter_sizes()\n",
    "se.param_sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dump_tensors(gpu_only=True):\n",
    "\t\"\"\"Prints a list of the Tensors being tracked by the garbage collector.\"\"\"\n",
    "\timport gc\n",
    "\ttotal_size = 0\n",
    "\tfor obj in tqdm_notebook(gc.get_objects()):\n",
    "\t\ttry:\n",
    "\t\t\tif torch.is_tensor(obj):\n",
    "\t\t\t\tif not gpu_only or obj.is_cuda:\n",
    "\t\t\t\t\tprint(\"%s:%s%s %s\" % (type(obj).__name__, \n",
    "\t\t\t\t\t\t\t\t\t\t  \" GPU\" if obj.is_cuda else \"\",\n",
    "\t\t\t\t\t\t\t\t\t\t  \" pinned\" if obj.is_pinned else \"\",\n",
    "\t\t\t\t\t\t\t\t\t\t  pretty_size(obj.size())))\n",
    "\t\t\t\t\ttotal_size += obj.numel()\n",
    "\t\t\telif hasattr(obj, \"data\") and torch.is_tensor(obj.data):\n",
    "\t\t\t\tif not gpu_only or obj.is_cuda:\n",
    "\t\t\t\t\tprint(\"%s → %s:%s%s%s%s %s\" % (type(obj).__name__, \n",
    "\t\t\t\t\t\t\t\t\t\t\t\t   type(obj.data).__name__, \n",
    "\t\t\t\t\t\t\t\t\t\t\t\t   \" GPU\" if obj.is_cuda else \"\",\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t   \" pinned\" if obj.data.is_pinned else \"\",\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t   \" grad\" if obj.requires_grad else \"\", \n",
    "\t\t\t\t\t\t\t\t\t\t\t\t   \" volatile\" if obj.volatile else \"\",\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t   pretty_size(obj.data.size())))\n",
    "\t\t\t\t\ttotal_size += obj.data.numel()\n",
    "\t\texcept Exception as e:\n",
    "\t\t\tpass        \n",
    "\tprint(\"Total size:\", total_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dump_tensors()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fakenews",
   "language": "python",
   "name": "fakenews"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
